{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from cot import Collection\n",
    "from cot.stats import evaluation_as_table"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# join strings from a list with underscore\n",
    "def join_strings(list_of_strings):\n",
    "    if list_of_strings is None:\n",
    "        list_of_strings = [\"None\"]\n",
    "    if isinstance(list_of_strings, str):\n",
    "        list_of_strings = [list_of_strings]\n",
    "    joined_string = \"\"\n",
    "    for string in list_of_strings:\n",
    "        joined_string += (\"-\" + str(string))\n",
    "    # delete first underscore\n",
    "    joined_string = joined_string[1:]\n",
    "    return(joined_string)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### generate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /home/kon/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "coll = Collection.from_json(\"ts_67_empty\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "| Name           | Train   | Valid   | Test   |\n",
       "|----------------|---------|---------|--------|\n",
       "| commonsense_qa | -       | 67      | -      |\n",
       "| med_qa         | -       | -       | 67     |\n",
       "| medmc_qa       | -       | 67      | -      |\n",
       "| open_book_qa   | -       | -       | 67     |\n",
       "| strategy_qa    | 67      | -       | -      |\n",
       "| worldtree      | -       | -       | 67     |\n",
       "\n",
       "Not loaded: ['aqua', 'asdiv', 'entailment_bank', 'gsm8k', 'mawps', 'pubmed_qa', 'qed', 'svamp']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Configuration of the input and parameters of the language model \n",
    "config={\n",
    "    \"instruction_keys\": None,\n",
    "    \"cot_trigger_keys\": [None, \"zhou-01\"],\n",
    "    \"answer_extraction_keys\": 'auto-kojima', \n",
    "    \"author\" : \"thoughtsource\",\n",
    "    \"api_service\": \"openai_chat\",\n",
    "    \"api_time_interval\": 1,\n",
    "    \"engine\": \"gpt-4\", \n",
    "    \"temperature\": 0,\n",
    "    \"max_tokens\": 512,\n",
    "    \"verbose\": False,\n",
    "    \"warn\": False,\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating commonsense_qa...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68c41995f8b04102a5fe8d937449bb99",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: HTTP code 502 from API (<!DOCTYPE html>\n",
      "<!--[if lt IE 7]> <html class=\"no-js ie6 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 7]>    <html class=\"no-js ie7 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 8]>    <html class=\"no-js ie8 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if gt IE 8]><!--> <html class=\"no-js\" lang=\"en-US\"> <!--<![endif]-->\n",
      "<head>\n",
      "\n",
      "\n",
      "<title>api.openai.com | 502: Bad gateway</title>\n",
      "<meta charset=\"UTF-8\" />\n",
      "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge\" />\n",
      "<meta name=\"robots\" content=\"noindex, nofollow\" />\n",
      "<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\" />\n",
      "<link rel=\"stylesheet\" id=\"cf_styles-css\" href=\"/cdn-cgi/styles/main.css\" />\n",
      "\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"cf-wrapper\">\n",
      "\n",
      "    \n",
      "\n",
      "    <div id=\"cf-error-details\" class=\"p-0\">\n",
      "        <header class=\"mx-auto pt-10 lg:pt-6 lg:px-8 w-240 lg:w-full mb-8\">\n",
      "            <h1 class=\"inline-block sm:block sm:mb-2 font-light text-60 lg:text-4xl text-black-dark leading-tight mr-2\">\n",
      "              \n",
      "              <span class=\"inline-block\">Bad gateway</span>\n",
      "              <span class=\"code-label\">Error code 502</span>\n",
      "            </h1>\n",
      "            <div>\n",
      "               Visit <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">cloudflare.com</a> for more information.\n",
      "            </div>\n",
      "            <div class=\"mt-3\">2023-04-12 09:03:10 UTC</div>\n",
      "        </header>\n",
      "        \n",
      "        <div class=\"my-8 bg-gradient-gray\">\n",
      "            <div class=\"w-240 lg:w-full mx-auto\">\n",
      "                <div class=\"clearfix md:px-8\">\n",
      "                  \n",
      "<div id=\"cf-browser-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-browser block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">You</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Browser\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-cloudflare-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    <span class=\"cf-icon-cloud block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    </a>\n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">Vienna</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    Cloudflare\n",
      "    </a>\n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-host-status\" class=\"cf-error-source relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-server block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-error w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">api.openai.com</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Host\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-red-error\">Error</span>\n",
      "</div>\n",
      "\n",
      "                </div>\n",
      "              \n",
      "            </div>\n",
      "        </div>\n",
      "\n",
      "        <div class=\"w-240 lg:w-full mx-auto mb-8 lg:px-8\">\n",
      "            <div class=\"clearfix\">\n",
      "                <div class=\"w-1/2 md:w-full float-left pr-6 md:pb-10 md:pr-0 leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What happened?</h2>\n",
      "                    <p>The web server reported a bad gateway error.</p>\n",
      "                </div>\n",
      "              \n",
      "                <div class=\"w-1/2 md:w-full float-left leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What can I do?</h2>\n",
      "                    <p class=\"mb-6\">Please try again in a few minutes.</p>\n",
      "                </div>\n",
      "            </div>\n",
      "              \n",
      "        </div>\n",
      "\n",
      "        <div class=\"cf-error-footer cf-wrapper w-240 lg:w-full py-10 sm:py-4 sm:px-8 mx-auto text-center sm:text-left border-solid border-0 border-t border-gray-300\">\n",
      "  <p class=\"text-13\">\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\">Cloudflare Ray ID: <strong class=\"font-semibold\">7b6a43d15bafc28c</strong></span>\n",
      "    <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    <span id=\"cf-footer-item-ip\" class=\"cf-footer-item hidden sm:block sm:mb-1\">\n",
      "      Your IP:\n",
      "      <button type=\"button\" id=\"cf-footer-ip-reveal\" class=\"cf-footer-ip-reveal-btn\">Click to reveal</button>\n",
      "      <span class=\"hidden\" id=\"cf-footer-ip\">193.171.177.145</span>\n",
      "      <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    </span>\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\"><span>Performance &amp; security by</span> <a rel=\"noopener noreferrer\" href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" id=\"brand_link\" target=\"_blank\">Cloudflare</a></span>\n",
      "    \n",
      "  </p>\n",
      "  <script>(function(){function d(){var b=a.getElementById(\"cf-footer-item-ip\"),c=a.getElementById(\"cf-footer-ip-reveal\");b&&\"classList\"in b&&(b.classList.remove(\"hidden\"),c.addEventListener(\"click\",function(){c.classList.add(\"hidden\");a.getElementById(\"cf-footer-ip\").classList.remove(\"hidden\")}))}var a=document;document.addEventListener&&a.addEventListener(\"DOMContentLoaded\",d)})();</script>\n",
      "</div><!-- /.error-footer -->\n",
      "\n",
      "\n",
      "    </div>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n",
      "\n",
      ").\n",
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: HTTP code 502 from API (<!DOCTYPE html>\n",
      "<!--[if lt IE 7]> <html class=\"no-js ie6 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 7]>    <html class=\"no-js ie7 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 8]>    <html class=\"no-js ie8 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if gt IE 8]><!--> <html class=\"no-js\" lang=\"en-US\"> <!--<![endif]-->\n",
      "<head>\n",
      "\n",
      "\n",
      "<title>api.openai.com | 502: Bad gateway</title>\n",
      "<meta charset=\"UTF-8\" />\n",
      "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge\" />\n",
      "<meta name=\"robots\" content=\"noindex, nofollow\" />\n",
      "<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\" />\n",
      "<link rel=\"stylesheet\" id=\"cf_styles-css\" href=\"/cdn-cgi/styles/main.css\" />\n",
      "\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"cf-wrapper\">\n",
      "\n",
      "    \n",
      "\n",
      "    <div id=\"cf-error-details\" class=\"p-0\">\n",
      "        <header class=\"mx-auto pt-10 lg:pt-6 lg:px-8 w-240 lg:w-full mb-8\">\n",
      "            <h1 class=\"inline-block sm:block sm:mb-2 font-light text-60 lg:text-4xl text-black-dark leading-tight mr-2\">\n",
      "              \n",
      "              <span class=\"inline-block\">Bad gateway</span>\n",
      "              <span class=\"code-label\">Error code 502</span>\n",
      "            </h1>\n",
      "            <div>\n",
      "               Visit <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">cloudflare.com</a> for more information.\n",
      "            </div>\n",
      "            <div class=\"mt-3\">2023-04-12 09:11:53 UTC</div>\n",
      "        </header>\n",
      "        \n",
      "        <div class=\"my-8 bg-gradient-gray\">\n",
      "            <div class=\"w-240 lg:w-full mx-auto\">\n",
      "                <div class=\"clearfix md:px-8\">\n",
      "                  \n",
      "<div id=\"cf-browser-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-browser block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">You</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Browser\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-cloudflare-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    <span class=\"cf-icon-cloud block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    </a>\n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">Vienna</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    Cloudflare\n",
      "    </a>\n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-host-status\" class=\"cf-error-source relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-server block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-error w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">api.openai.com</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Host\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-red-error\">Error</span>\n",
      "</div>\n",
      "\n",
      "                </div>\n",
      "              \n",
      "            </div>\n",
      "        </div>\n",
      "\n",
      "        <div class=\"w-240 lg:w-full mx-auto mb-8 lg:px-8\">\n",
      "            <div class=\"clearfix\">\n",
      "                <div class=\"w-1/2 md:w-full float-left pr-6 md:pb-10 md:pr-0 leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What happened?</h2>\n",
      "                    <p>The web server reported a bad gateway error.</p>\n",
      "                </div>\n",
      "              \n",
      "                <div class=\"w-1/2 md:w-full float-left leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What can I do?</h2>\n",
      "                    <p class=\"mb-6\">Please try again in a few minutes.</p>\n",
      "                </div>\n",
      "            </div>\n",
      "              \n",
      "        </div>\n",
      "\n",
      "        <div class=\"cf-error-footer cf-wrapper w-240 lg:w-full py-10 sm:py-4 sm:px-8 mx-auto text-center sm:text-left border-solid border-0 border-t border-gray-300\">\n",
      "  <p class=\"text-13\">\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\">Cloudflare Ray ID: <strong class=\"font-semibold\">7b6a50910af3c28c</strong></span>\n",
      "    <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    <span id=\"cf-footer-item-ip\" class=\"cf-footer-item hidden sm:block sm:mb-1\">\n",
      "      Your IP:\n",
      "      <button type=\"button\" id=\"cf-footer-ip-reveal\" class=\"cf-footer-ip-reveal-btn\">Click to reveal</button>\n",
      "      <span class=\"hidden\" id=\"cf-footer-ip\">193.171.177.145</span>\n",
      "      <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    </span>\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\"><span>Performance &amp; security by</span> <a rel=\"noopener noreferrer\" href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" id=\"brand_link\" target=\"_blank\">Cloudflare</a></span>\n",
      "    \n",
      "  </p>\n",
      "  <script>(function(){function d(){var b=a.getElementById(\"cf-footer-item-ip\"),c=a.getElementById(\"cf-footer-ip-reveal\");b&&\"classList\"in b&&(b.classList.remove(\"hidden\"),c.addEventListener(\"click\",function(){c.classList.add(\"hidden\");a.getElementById(\"cf-footer-ip\").classList.remove(\"hidden\")}))}var a=document;document.addEventListener&&a.addEventListener(\"DOMContentLoaded\",d)})();</script>\n",
      "</div><!-- /.error-footer -->\n",
      "\n",
      "\n",
      "    </div>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n",
      "\n",
      ").\n",
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 9c824984895f4353b19c537f36fcf068 in your message.).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating med_qa...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d385dbb4b0a14c9385fd5a21b9d2cbe3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID af374c70a61fa94eec1c2dbf5d5248b9 in your message.).\n",
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 28e68d494f3b6b40084c6626c3f61f7d in your message.).\n",
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: HTTP code 502 from API (<!DOCTYPE html>\n",
      "<!--[if lt IE 7]> <html class=\"no-js ie6 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 7]>    <html class=\"no-js ie7 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 8]>    <html class=\"no-js ie8 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if gt IE 8]><!--> <html class=\"no-js\" lang=\"en-US\"> <!--<![endif]-->\n",
      "<head>\n",
      "\n",
      "\n",
      "<title>api.openai.com | 502: Bad gateway</title>\n",
      "<meta charset=\"UTF-8\" />\n",
      "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge\" />\n",
      "<meta name=\"robots\" content=\"noindex, nofollow\" />\n",
      "<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\" />\n",
      "<link rel=\"stylesheet\" id=\"cf_styles-css\" href=\"/cdn-cgi/styles/main.css\" />\n",
      "\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"cf-wrapper\">\n",
      "\n",
      "    \n",
      "\n",
      "    <div id=\"cf-error-details\" class=\"p-0\">\n",
      "        <header class=\"mx-auto pt-10 lg:pt-6 lg:px-8 w-240 lg:w-full mb-8\">\n",
      "            <h1 class=\"inline-block sm:block sm:mb-2 font-light text-60 lg:text-4xl text-black-dark leading-tight mr-2\">\n",
      "              \n",
      "              <span class=\"inline-block\">Bad gateway</span>\n",
      "              <span class=\"code-label\">Error code 502</span>\n",
      "            </h1>\n",
      "            <div>\n",
      "               Visit <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">cloudflare.com</a> for more information.\n",
      "            </div>\n",
      "            <div class=\"mt-3\">2023-04-12 10:45:23 UTC</div>\n",
      "        </header>\n",
      "        \n",
      "        <div class=\"my-8 bg-gradient-gray\">\n",
      "            <div class=\"w-240 lg:w-full mx-auto\">\n",
      "                <div class=\"clearfix md:px-8\">\n",
      "                  \n",
      "<div id=\"cf-browser-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-browser block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">You</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Browser\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-cloudflare-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    <span class=\"cf-icon-cloud block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    </a>\n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">Vienna</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    Cloudflare\n",
      "    </a>\n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-host-status\" class=\"cf-error-source relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-server block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-error w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">api.openai.com</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Host\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-red-error\">Error</span>\n",
      "</div>\n",
      "\n",
      "                </div>\n",
      "              \n",
      "            </div>\n",
      "        </div>\n",
      "\n",
      "        <div class=\"w-240 lg:w-full mx-auto mb-8 lg:px-8\">\n",
      "            <div class=\"clearfix\">\n",
      "                <div class=\"w-1/2 md:w-full float-left pr-6 md:pb-10 md:pr-0 leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What happened?</h2>\n",
      "                    <p>The web server reported a bad gateway error.</p>\n",
      "                </div>\n",
      "              \n",
      "                <div class=\"w-1/2 md:w-full float-left leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What can I do?</h2>\n",
      "                    <p class=\"mb-6\">Please try again in a few minutes.</p>\n",
      "                </div>\n",
      "            </div>\n",
      "              \n",
      "        </div>\n",
      "\n",
      "        <div class=\"cf-error-footer cf-wrapper w-240 lg:w-full py-10 sm:py-4 sm:px-8 mx-auto text-center sm:text-left border-solid border-0 border-t border-gray-300\">\n",
      "  <p class=\"text-13\">\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\">Cloudflare Ray ID: <strong class=\"font-semibold\">7b6ad98798abc28c</strong></span>\n",
      "    <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    <span id=\"cf-footer-item-ip\" class=\"cf-footer-item hidden sm:block sm:mb-1\">\n",
      "      Your IP:\n",
      "      <button type=\"button\" id=\"cf-footer-ip-reveal\" class=\"cf-footer-ip-reveal-btn\">Click to reveal</button>\n",
      "      <span class=\"hidden\" id=\"cf-footer-ip\">193.171.177.145</span>\n",
      "      <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    </span>\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\"><span>Performance &amp; security by</span> <a rel=\"noopener noreferrer\" href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" id=\"brand_link\" target=\"_blank\">Cloudflare</a></span>\n",
      "    \n",
      "  </p>\n",
      "  <script>(function(){function d(){var b=a.getElementById(\"cf-footer-item-ip\"),c=a.getElementById(\"cf-footer-ip-reveal\");b&&\"classList\"in b&&(b.classList.remove(\"hidden\"),c.addEventListener(\"click\",function(){c.classList.add(\"hidden\");a.getElementById(\"cf-footer-ip\").classList.remove(\"hidden\")}))}var a=document;document.addEventListener&&a.addEventListener(\"DOMContentLoaded\",d)})();</script>\n",
      "</div><!-- /.error-footer -->\n",
      "\n",
      "\n",
      "    </div>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n",
      "\n",
      ").\n",
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised APIError: HTTP code 502 from API (<!DOCTYPE html>\n",
      "<!--[if lt IE 7]> <html class=\"no-js ie6 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 7]>    <html class=\"no-js ie7 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if IE 8]>    <html class=\"no-js ie8 oldie\" lang=\"en-US\"> <![endif]-->\n",
      "<!--[if gt IE 8]><!--> <html class=\"no-js\" lang=\"en-US\"> <!--<![endif]-->\n",
      "<head>\n",
      "\n",
      "\n",
      "<title>api.openai.com | 502: Bad gateway</title>\n",
      "<meta charset=\"UTF-8\" />\n",
      "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=UTF-8\" />\n",
      "<meta http-equiv=\"X-UA-Compatible\" content=\"IE=Edge\" />\n",
      "<meta name=\"robots\" content=\"noindex, nofollow\" />\n",
      "<meta name=\"viewport\" content=\"width=device-width,initial-scale=1\" />\n",
      "<link rel=\"stylesheet\" id=\"cf_styles-css\" href=\"/cdn-cgi/styles/main.css\" />\n",
      "\n",
      "\n",
      "</head>\n",
      "<body>\n",
      "<div id=\"cf-wrapper\">\n",
      "\n",
      "    \n",
      "\n",
      "    <div id=\"cf-error-details\" class=\"p-0\">\n",
      "        <header class=\"mx-auto pt-10 lg:pt-6 lg:px-8 w-240 lg:w-full mb-8\">\n",
      "            <h1 class=\"inline-block sm:block sm:mb-2 font-light text-60 lg:text-4xl text-black-dark leading-tight mr-2\">\n",
      "              \n",
      "              <span class=\"inline-block\">Bad gateway</span>\n",
      "              <span class=\"code-label\">Error code 502</span>\n",
      "            </h1>\n",
      "            <div>\n",
      "               Visit <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">cloudflare.com</a> for more information.\n",
      "            </div>\n",
      "            <div class=\"mt-3\">2023-04-12 10:51:04 UTC</div>\n",
      "        </header>\n",
      "        \n",
      "        <div class=\"my-8 bg-gradient-gray\">\n",
      "            <div class=\"w-240 lg:w-full mx-auto\">\n",
      "                <div class=\"clearfix md:px-8\">\n",
      "                  \n",
      "<div id=\"cf-browser-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-browser block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">You</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Browser\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-cloudflare-status\" class=\" relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    <span class=\"cf-icon-cloud block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-ok w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    </a>\n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">Vienna</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    <a href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" target=\"_blank\" rel=\"noopener noreferrer\">\n",
      "    Cloudflare\n",
      "    </a>\n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-green-success\">Working</span>\n",
      "</div>\n",
      "\n",
      "<div id=\"cf-host-status\" class=\"cf-error-source relative w-1/3 md:w-full py-15 md:p-0 md:py-8 md:text-left md:border-solid md:border-0 md:border-b md:border-gray-400 overflow-hidden float-left md:float-none text-center\">\n",
      "  <div class=\"relative mb-10 md:m-0\">\n",
      "    \n",
      "    <span class=\"cf-icon-server block md:hidden h-20 bg-center bg-no-repeat\"></span>\n",
      "    <span class=\"cf-icon-error w-12 h-12 absolute left-1/2 md:left-auto md:right-0 md:top-0 -ml-6 -bottom-4\"></span>\n",
      "    \n",
      "  </div>\n",
      "  <span class=\"md:block w-full truncate\">api.openai.com</span>\n",
      "  <h3 class=\"md:inline-block mt-3 md:mt-0 text-2xl text-gray-600 font-light leading-1.3\">\n",
      "    \n",
      "    Host\n",
      "    \n",
      "  </h3>\n",
      "  <span class=\"leading-1.3 text-2xl text-red-error\">Error</span>\n",
      "</div>\n",
      "\n",
      "                </div>\n",
      "              \n",
      "            </div>\n",
      "        </div>\n",
      "\n",
      "        <div class=\"w-240 lg:w-full mx-auto mb-8 lg:px-8\">\n",
      "            <div class=\"clearfix\">\n",
      "                <div class=\"w-1/2 md:w-full float-left pr-6 md:pb-10 md:pr-0 leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What happened?</h2>\n",
      "                    <p>The web server reported a bad gateway error.</p>\n",
      "                </div>\n",
      "              \n",
      "                <div class=\"w-1/2 md:w-full float-left leading-relaxed\">\n",
      "                    <h2 class=\"text-3xl font-normal leading-1.3 mb-4\">What can I do?</h2>\n",
      "                    <p class=\"mb-6\">Please try again in a few minutes.</p>\n",
      "                </div>\n",
      "            </div>\n",
      "              \n",
      "        </div>\n",
      "\n",
      "        <div class=\"cf-error-footer cf-wrapper w-240 lg:w-full py-10 sm:py-4 sm:px-8 mx-auto text-center sm:text-left border-solid border-0 border-t border-gray-300\">\n",
      "  <p class=\"text-13\">\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\">Cloudflare Ray ID: <strong class=\"font-semibold\">7b6ae1da8aa1c28c</strong></span>\n",
      "    <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    <span id=\"cf-footer-item-ip\" class=\"cf-footer-item hidden sm:block sm:mb-1\">\n",
      "      Your IP:\n",
      "      <button type=\"button\" id=\"cf-footer-ip-reveal\" class=\"cf-footer-ip-reveal-btn\">Click to reveal</button>\n",
      "      <span class=\"hidden\" id=\"cf-footer-ip\">193.171.177.145</span>\n",
      "      <span class=\"cf-footer-separator sm:hidden\">&bull;</span>\n",
      "    </span>\n",
      "    <span class=\"cf-footer-item sm:block sm:mb-1\"><span>Performance &amp; security by</span> <a rel=\"noopener noreferrer\" href=\"https://www.cloudflare.com/5xx-error-landing?utm_source=errorcode_502&utm_campaign=api.openai.com\" id=\"brand_link\" target=\"_blank\">Cloudflare</a></span>\n",
      "    \n",
      "  </p>\n",
      "  <script>(function(){function d(){var b=a.getElementById(\"cf-footer-item-ip\"),c=a.getElementById(\"cf-footer-ip-reveal\");b&&\"classList\"in b&&(b.classList.remove(\"hidden\"),c.addEventListener(\"click\",function(){c.classList.add(\"hidden\");a.getElementById(\"cf-footer-ip\").classList.remove(\"hidden\")}))}var a=document;document.addEventListener&&a.addEventListener(\"DOMContentLoaded\",d)})();</script>\n",
      "</div><!-- /.error-footer -->\n",
      "\n",
      "\n",
      "    </div>\n",
      "</div>\n",
      "</body>\n",
      "</html>\n",
      "\n",
      ").\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating medmc_qa...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4a599471654944c092a88a4e892f309e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID f795997b398c7873e1aafd8bb4192dab in your message.).\n",
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID c40510276b4d189c7be770b8777328c4 in your message.).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating open_book_qa...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3462b8200b424954b024c901789e75ce",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID e52f2b3cb9e4f96ee7c8f7db477ad9c5 in your message.).\n",
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 71bdb19b13958bbcd6b307a5d7cbc1ca in your message.).\n",
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 21fa6e248a71d8b618b8345fc5c01912 in your message.).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating strategy_qa...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed85ac3e32e3484fbf053b34c364ba8f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Retrying langchain.chat_models.openai.ChatOpenAI.completion_with_retry.<locals>._completion_with_retry in 4.0 seconds as it raised RateLimitError: That model is currently overloaded with other requests. You can retry your request, or contact us through our help center at help.openai.com if the error persists. (Please include the request ID 099733bc1f2693279d6c33198e46a330 in your message.).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generating worldtree...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0367b9b3e2e742d9b53153a29370f21d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "coll.generate(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9043ccd538bf4ab486086cabd7923941",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e9e1792cf99f42ca9002777b2897371d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "19865acb181e4919a2a9a9c7be024de9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "99bb4753296d480a9e97ff06c990d40c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1d9d97f6c1d4b7788e98a6eb9c99adc",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2ca17ae3d74c4f1cba5c2a6250149ece",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/67 [00:00<?, ?ex/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "{'commonsense_qa': {'validation': {'accuracy': {'gpt-4': {'None_None_kojima-A-E': 0.820896,\n",
       "     'None_zhou-01_kojima-A-E': 0.791045}}}},\n",
       " 'med_qa': {'test': {'accuracy': {'gpt-4': {'None_None_kojima-A-E': 0.746269,\n",
       "     'None_zhou-01_kojima-A-E': 0.731343}}}},\n",
       " 'medmc_qa': {'validation': {'accuracy': {'gpt-4': {'None_None_kojima-A-D': 0.716418,\n",
       "     'None_zhou-01_kojima-A-D': 0.656716}}}},\n",
       " 'open_book_qa': {'test': {'accuracy': {'gpt-4': {'None_None_kojima-A-D': 0.925373,\n",
       "     'None_zhou-01_kojima-A-D': 0.925373}}}},\n",
       " 'strategy_qa': {'train': {'accuracy': {'gpt-4': {'None_None_kojima-yes-no': 0.865672,\n",
       "     'None_zhou-01_kojima-yes-no': 0.850746}}}},\n",
       " 'worldtree': {'test': {'accuracy': {'gpt-4': {'None_None_kojima-A-D': 1.0,\n",
       "     'None_zhou-01_kojima-A-D': 1.0}}}}}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "coll.evaluate()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2bab0aee0f1b455da8373c86fb50e38e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7e7090d4a3bd4d3284d6070e60e05d32",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "09fdf15a83e9477ead9ab0330bdcdd22",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f518343b19604a2584b2da86ae363398",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d172f7fb79d548bdab3b2f43f2f9f8f2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31363191cc5c4fe7a57cdce0924f9b3d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Creating json from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "coll.dump(\"thoughtsource_67\" + \"_\" + config['api_service'] + \"_\" + config['engine'].replace(\"/\", \"_\") + \"_\" + join_strings(config[\"instruction_keys\"]) + \"_\" + join_strings(config[\"cot_trigger_keys\"]) + \".json\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### merge into"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_67 = Collection.from_json(\"thoughtsource_67.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_67 = ts_67.merge(coll)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ts_67.dump(\"thoughtsource_67.json\")"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### evaluate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "coll = Collection.from_json(\"thoughtsource_67.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# coll.select_generated_cots(author='thoughtsource', model=[\"gpt-3.5-turbo\", \"flan-T5-xxl\"], cot_trigger=None, instruction=None)\n",
    "coll.select_generated_cots(author='thoughtsource')\n",
    "coll.select_generated_cots(model='gpt-3.5-turbo')\n",
    "# coll.select_generated_cots(cot_trigger=[None,'zhou-01'])\n",
    "# coll.select_generated_cots(instruction=[None])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "eval = coll.evaluate()\n",
    "table = evaluation_as_table(eval)\n",
    "table"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "a19cb823e24c98ee05a9cfa4a3a579b5d56d4c1a735f2a12456750b95a1e155e"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
